# coding: utf-8
import requests 
from lxml import etree
import pandas as pd

max_page  = 54
all_clas,all_time,all_new,all_up_down,all_up,all_down,all_num,all_money,all_end_price= [list() for x in range(9)]
headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/85.0.4183.102 Safari/537.36'}  #构造头文件，模拟浏览器。
for page in range(1,max_page+1):
    print('crawling the page is {}'.format(page))
    url= f'http://www.hbets.cn/list/13.html?page={page}'
    response  = requests.get(url, headers=headers) 
    # if response.status_code == 200:
    #     print('200')
    root = etree.HTML(response.text)  
    clas = root.xpath('//div[@class="future_table"]//li[@class="li1"]/text()')[1::]
    time = root.xpath('//div[@class="future_table"]//li[@class="li2"]/text()')[1::]
    new = root.xpath('//div[@class="future_table"]//li[@class="li3"]/text()')[1::]
    up_down = root.xpath('//div[@class="future_table"]//li[@class="li4"]/text()')[1::]
    up = root.xpath('//div[@class="future_table"]//li[@class="li5"]/text()')[1::]
    down = root.xpath('//div[@class="future_table"]//li[@class="li6"]/text()')[1::]
    num = root.xpath('//div[@class="future_table"]//li[@class="li7"]/text()')[1::]
    money = root.xpath('//div[@class="future_table"]//li[@class="li8"]/text()')[1::]
    end_price = root.xpath('//div[@class="future_table"]//li[@class="li9"]/text()')[1::]

    all_clas+=clas
    all_time+=time
    all_new+=new
    all_up_down+=up_down
    all_up+=up
    all_down+=down
    all_num+=num
    all_money+=money
    all_end_price+=end_price


data_raw = pd.DataFrame()  
data_raw['交易品种'] = all_clas
data_raw['日期'] = all_time
data_raw['最新'] = all_new
data_raw['涨跌幅'] = all_up_down
data_raw['最高'] = all_up
data_raw['最低'] = all_down
data_raw['成交量'] = all_num
data_raw['成交额'] = all_money
data_raw['昨收盘价'] = all_end_price
data_raw.to_excel('.//湖北碳排放交易中心现货交易数据每日概况.xlsx', index=False)  